#coding=utf-8
#!/usr/bin/python

import urllib2

# personal lib
import SpUtil
import SpParser
import SpRegExpressions

# some  var declear
url = 'http://cb.1024hgc.com/bbs/forum-3-2.html'
htmlPath = "html.txt"
hrefPath = "href.txt"
hrefAndDescPath = "hrefAndDesc.txt"


americanEurop = []
japanCode =[]
japanNoCode =[]
asian = []
asianCode=[]
asianNoCode=[]
otherJumpObjs=[]


debugStep1Request = True
debugStep2WriteFile =True

# 伪装成浏览器
req_header = {
'User-Agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
# 'Accept':'text/html;q=0.9,*/*;q=0.8',
# 'Accept-Charset':'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
# 'Accept-Encoding':'gzip',
# 'Connection':'close',
# 'Referer':None #注意如果依然不能抓取的话，这里可以设置抓取网站的host
}

# 1. request url
request = urllib2.Request(url,None,req_header)
response = urllib2.urlopen(request)
page=response.read()
# print page

# 2.write to local path
SpUtil.SpWrite(htmlPath,page)

# 3.parse html and write to href path
parser = SpParser.parserXml()
parser.feed(page)
SpParser.SpWriteHrefFile(hrefPath,parser)
SpParser.SpWriteHrefAndDescFile(hrefAndDescPath,parser)

#


re = SpRegExpressions.SpRE()
jumpobjs = parser.jumpObj
for obj in jumpobjs:
    if re.execRE(r'(.)*歐美(.)*',obj.jumpDesc)==True:
        americanEurop.append(obj)
    elif re.execRE(r'(.)*亞洲無碼(.)*',obj.jumpDesc)==True:
        asianNoCode.append(obj)
    elif re.execRE(r'(.)*亞洲騎兵(.)*',obj.jumpDesc)==True:
        asianCode.append(obj)
    elif re.execRE(r'(.)*日本騎兵(.)*',obj.jumpDesc)==True:
        japanCode.append(obj)
    elif re.execRE(r'(.)*日本無碼(.)*',obj.jumpDesc)==True:
        japanNoCode.append(obj)
    else:
        otherJumpObjs.append(obj)
parser.close()





# print request.get_host()











ameJmpObj = americanEurop[0]
# print  ameJmpObj.jumpUrl,"\n",ameJmpObj.jumpDesc

realURL ="http://cb.1024hgc.com/bbs/"+ameJmpObj.jumpUrl[0]
print realURL

savepath = ameJmpObj.jumpDesc+".html"

request = urllib2.Request(realURL,None,req_header)
response = urllib2.urlopen(request)
page=response.read()
print  page

SpUtil.SpWrite(savepath,page)